
#include <boost/algorithm/string.hpp>
#include "WebKit.h"
#include "WebPage.h"
#include "WebElement.h"
#include "thread_spider.h"

using namespace SpiderWebKitClient;
using namespace std;
using namespace boost::algorithm;

#define BAIJIU_URL "http://list.jiuxian.com/1-412-0-0-0-0-0-0-0-0-0-0-0-0-0-0.htm#v2"
#define PUTAOJIU_URL "http://list.jiuxian.com/2-0-0-0-0-0-0-0-0-0-0-0.htm#v2"
#define YANGJIU_URL "http://list.jiuxian.com/4-0-0-0-0-0-0-0-0-0-0-0.htm#v2"
#define BAOJIANJIU_URL "http://list.jiuxian.com/6-0-0-0-0-0-0-0-0-0-0-0.htm#v2"
#define HUANGJIU_URL "http://list.jiuxian.com/95-0-0-0-0-0-0-0-0-0-0-0.htm#v2"
#define PIJIU_URL "http://list.jiuxian.com/151-0-0-0-0-0-0-0-0-0-0-0.htm#v2"

#define PRODUCT_LINK_CSS "body > div.srp.clearfix > div.main-srp > div.proListSearch > ul > li > div.content.clearfix > a.proName"
#define PRODUCT_NEXTPAGE_LINK_CSS "body > div.srp.clearfix > div.main-srp > div > div > a.nextpage"
#define PRODUCT_NEXTPAGE_HREF_NONE "javascript:void(0)"

static long jiu_product_count = 0;
class JiuxianParser
{
    public:
    JiuxianParser()
        : m_finshed(false)
    {
        std::list<std::string> urls;
        urls.push_back(BAIJIU_URL);
        urls.push_back(PUTAOJIU_URL);
        urls.push_back(YANGJIU_URL);
        urls.push_back(BAOJIANJIU_URL);
        urls.push_back(HUANGJIU_URL);
        urls.push_back(PIJIU_URL);

        std::list<std::string>::iterator it;
        for (it = urls.begin(); it != urls.end(); it ++) {
            WKRequest* req = m_spider.createRequest(*it, this, &JiuxianParser::parse_product_page, 102);
            //req->start();
        }
        m_spider.start();
    }

    void start()
    {
        while(1) {
            if (m_finshed)
                break;
            usleep(500000);
        }
    }

    void parse_product_page(WebPage *page)
    {
        std::cout << "curr page:" << page->getTitle() << std::endl;
        std::cout << "curr page:" << page->getUrl() << std::endl;
        std::list<WebElement> nextCollections = page->getElementsByCssSelector(PRODUCT_NEXTPAGE_LINK_CSS);
        if (nextCollections.size() > 0) {
            WebElement element = nextCollections.front();
            std::string url = element.getAttribute("href");
            if (url != "" && url != PRODUCT_NEXTPAGE_HREF_NONE) {
                if (!boost::starts_with(url, "http://")) {
                    std::string indexUrl = page->getUrl();
                    std::string subUrl = indexUrl.substr(7);
                    std::string domain = "http://" + subUrl.substr(0, subUrl.find_first_of("/")) + "/";
                    url = domain + url;
                }
                std::cout << "next page:" << url << std::endl;
                WKRequest* req = m_spider.createRequest(url, this, &JiuxianParser::parse_product_page, 101);
            }
        }

        std::list<WebElement> collection = page->getElementsByCssSelector(PRODUCT_LINK_CSS);

        boost::mutex::scoped_lock lock(m_countMutex);
        if (collection.size() > 0) {
            jiu_product_count += collection.size();
            printf("Product Cout : %ld\n", jiu_product_count);
        }
        if (m_spider.requestCount() == 0)
            m_finshed = true;
    }

    private:
    ThreadSpider m_spider;
    boost::mutex m_countMutex;
    bool m_finshed;

};

void collectJiuxian()
{
    JiuxianParser p;
    p.start();
}

